﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.XPath;
using System.IO;

namespace GateExtractor
{
    class Cluster
    {
        public List<string> ids = new List<string>();
        public Dictionary<string, double> GetAvgAttributes(Dictionary<string, Dictionary<string, double>> elements)
        {
            Dictionary<string, double> attrs = new Dictionary<string, double>();
            foreach (string key in ids)
            {
                foreach (KeyValuePair<string, double> kvpAttribute in elements[key])
                {
                    if (attrs.ContainsKey(kvpAttribute.Key))
                        attrs[kvpAttribute.Key] += kvpAttribute.Value / ids.Count;
                    else
                        attrs.Add(kvpAttribute.Key, kvpAttribute.Value / ids.Count);
                }
            }
            return attrs;
        }
        public string GetAvgAttributesStr(Dictionary<string, Dictionary<string, double>> elements)
        {
            Dictionary<string, double> attrs = GetAvgAttributes(elements);
            string strResult="";
            //dispersion
            double sum = 0;
            foreach (KeyValuePair<string, double> kvpAttribute in attrs)
            {
                double tempVal = 0;
                foreach (string key in ids)
                {
                    if (elements[key].ContainsKey(kvpAttribute.Key))
                        tempVal = elements[key][kvpAttribute.Key];
                    sum += (tempVal - kvpAttribute.Value) * (tempVal - kvpAttribute.Value);
                }
                double disp = sum / ids.Count;
                strResult += kvpAttribute.Key + "=" + kvpAttribute.Value + "(disp:"+disp+")   ";
            }

            
            return strResult;
        }
        public static double GetDistance(Cluster cl1, Cluster cl2, Dictionary<string, Dictionary<string, double>> elements) //euclidian
        {
            double dist = 0;
            List<string> attrNames= new List<string>();
            Dictionary<string, double> attrs1 = cl1.GetAvgAttributes(elements);
            Dictionary<string, double> attrs2 = cl2.GetAvgAttributes(elements);

            foreach (string key in attrs1.Keys)
                if (!attrNames.Contains(key)) attrNames.Add(key);
            foreach (string key in attrs2.Keys)
                if (!attrNames.Contains(key)) attrNames.Add(key);

            foreach (string key in attrNames)
            {
                double val1 = 0;
                if (attrs1.ContainsKey(key)) val1 = attrs1[key];
                double val2 = 0;
                if (attrs2.ContainsKey(key)) val2 = attrs2[key];
                dist += (val2 - val1)*(val2-val1);
            }
            return dist;
        }

    }

    class Processor
    {



        XmlNodeList nodesSyntaxTreeNodes;
        //per file stats:
        Dictionary<string, Dictionary<string, int>> personages;
        Dictionary<string, Dictionary<string, double>> personagesNormalized;

        public XmlNode GetSyntaxTreeNode(int id)
        {
            //  Console.WriteLine("searching for "+id.ToString());
            foreach (XmlNode currentNode in nodesSyntaxTreeNodes)
            {

                XmlNode feature = currentNode.SelectSingleNode("Feature");
                XmlNode val = feature.SelectSingleNode("Value");
                //Console.WriteLine(val.InnerText);
                if (System.Convert.ToInt32(val.InnerText) == id) return currentNode;
            }
            Console.WriteLine("missing syntax tree node");
            return null;
        }



        public void Process()
        {
            Console.WriteLine("GateExtractor 0.2");
            DateTime dt_start = DateTime.Now;

            //global init

            personagesNormalized = new Dictionary<string, Dictionary<string, double>>();

            DirectoryInfo di = new DirectoryInfo("./in");
            FileInfo[] Files = di.GetFiles();
            try
            {
                if (Directory.Exists("./out")) Directory.Delete("./out", true);
            }
            catch (Exception ex)
            {
                Console.WriteLine("./out :" + ex.Message);
            }
            try { Directory.CreateDirectory("./out"); }
            catch (Exception ex) { Console.WriteLine("./out :" + ex.Message); };

            foreach (FileInfo fi in Files)
            {
                ProcessFile(fi);
            }
            //global results
            Console.WriteLine("---------------Clustering----------");
            Clusterize();
        }


        void PrintSyntaxTreeNode(int id, TextWriter wr)
        {

            XmlNode nodeTreeNode = GetSyntaxTreeNode(id);
            if (nodeTreeNode != null)
            {
                //wr.WriteLine(nodeTreeNode.InnerXml);
                wr.Write(nodeTreeNode.SelectNodes("Feature")[2].SelectSingleNode("Value").InnerText);
                wr.Write("; ");
            }
            else
            {
                wr.Write("!!!missing!!! ;");
            }

        }
        void Clusterize()
        {
            TextWriter wr = new StreamWriter("./out/___clusters");
            List<Cluster> clusters = new List<Cluster>();
            foreach (KeyValuePair<string, Dictionary<string, double>> kvpPersons in personagesNormalized)
            {
                wr.Write(kvpPersons.Key + "\t" + kvpPersons.Value.Count + ":\t");

                foreach (KeyValuePair<string, double> kvpAttribute in kvpPersons.Value)
                {
                    wr.Write(kvpAttribute.Key + " = " + kvpAttribute.Value.ToString() + "\t");
                }
                wr.WriteLine();
                Cluster newCluster = new Cluster();
                newCluster.ids.Add(kvpPersons.Key);
                clusters.Add(newCluster);
            }
            wr.WriteLine();
            //create singular classes
            int pass = 0;
            while (clusters.Count>1)
            {
                //print current clusters
                wr.WriteLine("\n#######################      pass " + pass.ToString() + "      #########################");
                wr.WriteLine("clusters:");
                foreach (Cluster curCluster in clusters)
                {
                    wr.Write("*\t");
                    foreach (string key in curCluster.ids)
                    {
                        wr.Write(key + "   ");
                    }
                    wr.WriteLine();
                    // print average attributes
                    wr.WriteLine("\taverage attributes: " + curCluster.GetAvgAttributesStr(personagesNormalized));
                    wr.WriteLine();
                }
                //selecting closest clusters
                
                List<int> idxClustersToJoin= new List<int>();
                double distMin = Cluster.GetDistance(clusters[0], clusters[1], personagesNormalized);;
                idxClustersToJoin.Add(0);
                idxClustersToJoin.Add(1);

                for (int i = 0; i < clusters.Count - 1; i++)
                    for (int j = i + 1; j < clusters.Count; j++)
                    {
                        double dist = Cluster.GetDistance(clusters[i], clusters[j], personagesNormalized);
                        if (dist < distMin)
                        {
                            idxClustersToJoin[0] = i;
                            idxClustersToJoin[1] = j;
                            distMin = dist;
                        }
                    }
                
                
                
                //print clusters to join
                wr.WriteLine("will be joint:");
                foreach (int idx in idxClustersToJoin)
                {
                    wr.Write("*\t");
                    foreach (string key in clusters[idx].ids)
                    {
                        wr.Write(key + "  ");
                    }
                    wr.WriteLine();
                    wr.WriteLine(clusters[idx].GetAvgAttributesStr(personagesNormalized));
                    //write attrs
                    wr.WriteLine();
                }
                wr.WriteLine("dist=" + distMin);
                //joining closest clusters
                List<Cluster> newClusters = new List<Cluster>();
                for (int i=0;i<clusters.Count;i++) //moving old clusters
                {
                    if (!idxClustersToJoin.Contains(i))
                    {
                        newClusters.Add(clusters[i]);
                    }
                } 
                //merging clusters
                Cluster mergedCluster = new Cluster();
                foreach (int id in idxClustersToJoin)
                {
                    foreach (string key in clusters[id].ids)
                        mergedCluster.ids.Add(key);
                }
                newClusters.Add(mergedCluster);
                clusters = newClusters;
                pass++;
            }
            

            wr.Close();
        }
        void ProcessFile(FileInfo fi)
        {
            Console.WriteLine("\nProcessing " + fi.Name);
            int numWords = 0;
            TextWriter wr = new StreamWriter("./out/" + fi.Name);

            try
            {

                XmlDocument myDoc = new XmlDocument();
                //init
                personages = new Dictionary<string, Dictionary<string, int>>();
                // nodesSyntaxTreeNodes = myDoc.SelectNodes("//Annotation[@Type='SyntaxTreeNode']");
                //load text
                myDoc.PreserveWhitespace = true;
                myDoc.Load(fi.FullName);
                {
                    XmlNodeList nodes = myDoc.SelectNodes("//Node");
                    numWords = nodes.Count;
                }

                XmlNode text = myDoc.SelectSingleNode("//TextWithNodes");

                // text.RemoveAll();
                string strText = text.InnerText;
                //wr.Write(text.InnerText);

                //load features
                XmlNodeList nodesNess = myDoc.SelectNodes("//Annotation[@Type='n']");
                // XmlNodeList nodesDep = myDoc.SelectNodes("//Annotation[@Type='Dependency']");
                Console.WriteLine(nodesNess.Count.ToString() + " ness annotations selected");
                for (int i = 0; i < nodesNess.Count; i++)
                {
                    //find start|end nodes
                    wr.WriteLine("######################################################################################################################");
                    int posStartNess = System.Convert.ToInt32(nodesNess[i].Attributes["StartNode"].Value);
                    int posEndNess = System.Convert.ToInt32(nodesNess[i].Attributes["EndNode"].Value);
                    string strNessText = strText.Substring(posStartNess, posEndNess - posStartNess);
                    wr.WriteLine("exploring ness " + strNessText);

                    //print ness props
                    string carrier = "";
                    string sem = "";
                    string polarity = "";
                    foreach (XmlNode nessFeature in nodesNess[i].SelectNodes("Feature"))
                    {
                        wr.Write(nessFeature.SelectSingleNode("Name").InnerText + ":\t");
                        wr.WriteLine(nessFeature.SelectSingleNode("Value").InnerText);
                        if (nessFeature.SelectSingleNode("Name").InnerText == "carrier")
                            carrier = nessFeature.SelectSingleNode("Value").InnerText.ToLower();
                        if (nessFeature.SelectSingleNode("Name").InnerText == "qd")
                            carrier = nessFeature.SelectSingleNode("Value").InnerText.ToLower();
                        if (nessFeature.SelectSingleNode("Name").InnerText == "sem")
                            sem = nessFeature.SelectSingleNode("Value").InnerText;

                        if (nessFeature.SelectSingleNode("Name").InnerText == "polarity")
                            polarity = nessFeature.SelectSingleNode("Value").InnerText;
                        if (nessFeature.SelectSingleNode("Name").InnerText == "pol")
                            polarity = nessFeature.SelectSingleNode("Value").InnerText;

                    }

                    wr.WriteLine();
                    if (!personages.ContainsKey(carrier))
                        personages.Add(carrier, new Dictionary<string, int>());
                    Dictionary<string, int> peronageStats = personages[carrier];
                    string semConcatenated = "(" + polarity + ")" + sem;
                    if (peronageStats.ContainsKey(semConcatenated))
                        peronageStats[semConcatenated]++;
                    else
                        peronageStats.Add(semConcatenated, 1);
                    //    wr.WriteLine(nodesNess[i].InnerXml);
                    //searching annotation with this ness
                    /*
                    for (int j = 0; j < nodesDep.Count; j++)
                    {
                        int posStartDep = System.Convert.ToInt32(nodesDep[j].Attributes["StartNode"].Value);
                        int posEndDep = System.Convert.ToInt32(nodesDep[j].Attributes["EndNode"].Value);
                        if ((posEndNess >= posStartDep) && (posEndNess <= posEndDep)) //can be missing
                        {
                            string strDepText = strText.Substring(posStartDep, posEndDep - posStartDep);
                            wr.WriteLine("dependency: ");
                            //wr.WriteLine(nodesDep[j].OuterXml);
                            XmlNodeList features = nodesDep[j].SelectNodes("Feature");
                            string kind = features[1].SelectSingleNode("Value").InnerText;
                            wr.WriteLine("kind: " + kind);
                            wr.WriteLine("text: " + strDepText);

                            string args = features[0].SelectSingleNode("Value").InnerText;
                            string[] argTokens = args.Split(';');

                            /// wr.Write("args: " );
                            // foreach (string arg in argTokens)
                            //    PrintSyntaxTreeNode(System.Convert.ToInt32(arg),wr);

                            wr.WriteLine("\n");
                            //find arguments
                            //wr.WriteLine(nodesDep[j].OuterXml);
                        }
                    }
                     * */


                }//end processing nesses
                //printing personage info
                foreach (KeyValuePair<string, Dictionary<string, int>> kvpPersons in personages)
                {
                    wr.Write(kvpPersons.Key + "\t" + kvpPersons.Value.Count + ":\t");
                    string key = fi.Name.Substring(0, fi.Name.Length - 3) + kvpPersons.Key;
                    personagesNormalized.Add(key, new Dictionary<string, double>());
                    foreach (KeyValuePair<string, int> kvpAttribute in kvpPersons.Value)
                    {
                        double nessFreqMultiplier = 10000;
                        personagesNormalized[key].Add(kvpAttribute.Key, ((double)kvpAttribute.Value) / numWords * nessFreqMultiplier);
                        wr.Write(kvpAttribute.Key + "(" + kvpAttribute.Value.ToString() + ")\t");
                    }
                    wr.WriteLine();
                }


            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }


            wr.Close();


        }// end processfile
    }

}
